package indexer.processer.ybdu

import indexer.processer.UrlPageProcessor
import us.codecraft.webmagic.Site
import us.codecraft.webmagic.selector.Selectable

class YbduPageProcessor : UrlPageProcessor{
    override fun getUrl(): String = "http://www.ybdu.com/"

    override fun getSite(): Site {
        return Site.me().setRetryTimes(3).setSleepTime(1000).setCycleRetryTimes(3).setTimeOut(60_000).setRetrySleepTime(10_000)
    }

    override fun process(page: us.codecraft.webmagic.Page) {
        val html = page.html

        val urls = html.links().regex("https?://.*\\.ybdu.com.*").all()
        page.addTargetRequests(urls)

        val mainInfo: Selectable = html.xpath("//div[@class='mu_contain']")

        if (mainInfo.match()) {
            val name = html.xpath("//div[@class='mu_h1']/h1/text()").get()
            val author = ""
            val url = page.request.url
            val idx = name.indexOf("全文阅读")
            page.putField("name",name.substring(0,idx).trim())
            page.putField("url",url)
            page.putField("author",author)

        } else {
            page.setSkip(true)

        }


    }
}